{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\" style=\"margin-top: 1em;\"><ul class=\"toc-item\"></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected cross entropy loss if the model:\n",
      "- learns neither dependency: 0.661563238158\n",
      "- learns first dependency: 0.519166699707\n",
      "- learns both dependency: 0.454454367449\n"
     ]
    }
   ],
   "source": [
    "print(\"Expected cross entropy loss if the model:\")\n",
    "print(\"- learns neither dependency:\", -(0.625 * np.log(0.625) + 0.375 * np.log(0.375)))\n",
    "print(\"- learns first dependency:\", \n",
    "      -0.5 * (0.875 * np.log(0.875) + 0.125 * np.log(0.125))\n",
    "      -0.5 * (0.625 * np.log(0.625) + 0.375 * np.log(0.375)))\n",
    "print(\"- learns both dependency:\", \n",
    "      -0.5 * (0.75 * np.log(0.75) + 0.25 * np.log(0.25))\n",
    "      -0.25 * (2 * 0.5 * np.log(0.5) - 0.25 * (0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES']=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_steps = 5\n",
    "batch_size = 200\n",
    "num_classes = 2\n",
    "state_size = 16\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_data(size = 1000000):\n",
    "    X = np.array(np.random.choice(2, size=(size,)))\n",
    "    Y = []\n",
    "    for i in range(size):\n",
    "        threshold = 0.5\n",
    "        if X[i-3] == 1:\n",
    "            threshold += 0.5\n",
    "        if X[i-8] == 1:\n",
    "            threshold -= 0.25\n",
    "        if np.random.rand() > threshold:\n",
    "            Y.append(0)\n",
    "        else:\n",
    "            Y.append(1)\n",
    "    return X, np.array(Y)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_batch(raw_date, batch_size, num_steps):\n",
    "    raw_x, raw_y = raw_date\n",
    "    data_length = len(raw_x)\n",
    "    \n",
    "    #partition raw data into batches and stak them vertically in a data matrix\n",
    "    batch_partition_length = data_length // batch_size\n",
    "    data_x = np.zeros([batch_size, batch_partition_length], dtype=np.int32)\n",
    "    data_y = np.zeros([batch_size, batch_partition_length], dtype=np.int32)\n",
    "    # do partition \n",
    "    for i in range(batch_size):\n",
    "        data_x[i] = raw_x[batch_partition_length * i : batch_partition_length * (i + 1)]\n",
    "        data_y[i] = raw_y[batch_partition_length * i : batch_partition_length * (i + 1)]\n",
    "    # do epoch\n",
    "    epoch_size = batch_partition_length // num_steps\n",
    "    \n",
    "    for i in range(epoch_size):\n",
    "        x = data_x[:, i * num_steps:(i + 1) * num_steps]\n",
    "        y = data_y[:, i * num_steps:(i + 1) * num_steps]\n",
    "        yield(x, y)\n",
    "        \n",
    "def gen_epochs(n, num_steps):\n",
    "    for i in range(n):\n",
    "        yield gen_batch(gen_data(), batch_size, num_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "x = tf.placeholder(tf.int32, [batch_size, num_steps], name='input_placeholder')\n",
    "y = tf.placeholder(tf.int32, [batch_size, num_steps], name='labels_placeholder')\n",
    "init_state = tf.zeros([batch_size, state_size])\n",
    "\n",
    "x_one_hot = tf.one_hot(x, num_classes)\n",
    "rnn_inputs = tf.unstack(x_one_hot, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tf.Tensor 'rnn/rnn/basic_rnn_cell/Tanh:0' shape=(200, 16) dtype=float32>, <tf.Tensor 'rnn/rnn/basic_rnn_cell/Tanh_1:0' shape=(200, 16) dtype=float32>, <tf.Tensor 'rnn/rnn/basic_rnn_cell/Tanh_2:0' shape=(200, 16) dtype=float32>, <tf.Tensor 'rnn/rnn/basic_rnn_cell/Tanh_3:0' shape=(200, 16) dtype=float32>, <tf.Tensor 'rnn/rnn/basic_rnn_cell/Tanh_4:0' shape=(200, 16) dtype=float32>]\n",
      "Tensor(\"rnn/rnn/basic_rnn_cell/Tanh_4:0\", shape=(200, 16), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# with tf.variable_scope('rnn_cell'):\n",
    "#     W = tf.get_variable('W', [num_classes + state_size, state_size])\n",
    "#     b = tf.get_variable('b', [state_size], initializer=tf.constant_initializer(0.0))\n",
    "\n",
    "# def rnn_cell(rnn_input, state):\n",
    "#     with tf.variable_scope('rnn_cell', reuse=True):\n",
    "#         W = tf.get_variable('W', [num_classes + state_size, state_size])\n",
    "#         b = tf.get_variable('b', [state_size], initializer=tf.constant_initializer(0.0))\n",
    "#     return tf.tanh(tf.matmul(tf.concat([rnn_input, state], 1), W) + b)\n",
    "    \n",
    "# state = init_state\n",
    "# rnn_outputs = []\n",
    "# for rnn_input in rnn_inputs:\n",
    "#     state = rnn_cell(rnn_input, state)\n",
    "#     rnn_outputs.append(state)\n",
    "# final_state = rnn_outputs[-1]\n",
    "\n",
    "# 上面是原始代码，定义了rnn_cell，然后使用循环的方式对其进行复用，\n",
    "#利用tensorflow简化之后我们可以直接调用BasicRNNCell和static_rnn两个函数实现\n",
    "cell = tf.nn.rnn_cell.BasicRNNCell(state_size)\n",
    "rnn_outputs, final_state = tf.nn.static_rnn(cell, rnn_inputs, initial_state=init_state)\n",
    "print(rnn_outputs)\n",
    "print(final_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.variable_scope('softmax'):\n",
    "    W = tf.get_variable('W', [state_size, num_classes])\n",
    "    b = tf.get_variable('b', [num_classes], initializer=tf.constant_initializer(0.0))\n",
    "logits = [tf.matmul(rnn_output, W) + b for rnn_output in rnn_outputs]\n",
    "predictions = [tf.nn.softmax(logit) for logit in logits]\n",
    "\n",
    "y_as_list = tf.unstack(y, num=num_steps, axis=1)\n",
    "\n",
    "losses = [tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label, logits=logit) for \n",
    "         logit, label in zip(logits, y_as_list)]\n",
    "total_loss = tf.reduce_mean(losses)\n",
    "train_step = tf.train.AdamOptimizer(learning_rate).minimize(total_loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "EPOCH 0\n",
      "Average loss at step 100 for last 100 steps: 0.60592177242\n",
      "Average loss at step 200 for last 100 steps: 0.494117258489\n",
      "Average loss at step 300 for last 100 steps: 0.487235401273\n",
      "Average loss at step 400 for last 100 steps: 0.488104067743\n",
      "Average loss at step 500 for last 100 steps: 0.484612516463\n",
      "Average loss at step 600 for last 100 steps: 0.485257371664\n",
      "Average loss at step 700 for last 100 steps: 0.484761792421\n",
      "Average loss at step 800 for last 100 steps: 0.484488342404\n",
      "Average loss at step 900 for last 100 steps: 0.482228897214\n",
      "\n",
      "EPOCH 1\n",
      "Average loss at step 100 for last 100 steps: 0.490588644147\n",
      "Average loss at step 200 for last 100 steps: 0.484489214122\n",
      "Average loss at step 300 for last 100 steps: 0.483278390467\n",
      "Average loss at step 400 for last 100 steps: 0.479197616577\n",
      "Average loss at step 500 for last 100 steps: 0.480221810639\n",
      "Average loss at step 600 for last 100 steps: 0.477422969341\n",
      "Average loss at step 700 for last 100 steps: 0.479821939468\n",
      "Average loss at step 800 for last 100 steps: 0.480770166218\n",
      "Average loss at step 900 for last 100 steps: 0.477210979462\n",
      "\n",
      "EPOCH 2\n",
      "Average loss at step 100 for last 100 steps: 0.484877128601\n",
      "Average loss at step 200 for last 100 steps: 0.476121207178\n",
      "Average loss at step 300 for last 100 steps: 0.479795797169\n",
      "Average loss at step 400 for last 100 steps: 0.477835725248\n",
      "Average loss at step 500 for last 100 steps: 0.476721453369\n",
      "Average loss at step 600 for last 100 steps: 0.475965521336\n",
      "Average loss at step 700 for last 100 steps: 0.473901064694\n",
      "Average loss at step 800 for last 100 steps: 0.474355764687\n",
      "Average loss at step 900 for last 100 steps: 0.473182309866\n",
      "\n",
      "EPOCH 3\n",
      "Average loss at step 100 for last 100 steps: 0.481113934517\n",
      "Average loss at step 200 for last 100 steps: 0.470775363743\n",
      "Average loss at step 300 for last 100 steps: 0.470336705148\n",
      "Average loss at step 400 for last 100 steps: 0.47133379966\n",
      "Average loss at step 500 for last 100 steps: 0.470391560197\n",
      "Average loss at step 600 for last 100 steps: 0.470213287473\n",
      "Average loss at step 700 for last 100 steps: 0.466801314652\n",
      "Average loss at step 800 for last 100 steps: 0.468586094379\n",
      "Average loss at step 900 for last 100 steps: 0.467501805723\n",
      "\n",
      "EPOCH 4\n",
      "Average loss at step 100 for last 100 steps: 0.473757324517\n",
      "Average loss at step 200 for last 100 steps: 0.46723693192\n",
      "Average loss at step 300 for last 100 steps: 0.467234064043\n",
      "Average loss at step 400 for last 100 steps: 0.467709053457\n",
      "Average loss at step 500 for last 100 steps: 0.466509019434\n",
      "Average loss at step 600 for last 100 steps: 0.468824896216\n",
      "Average loss at step 700 for last 100 steps: 0.466975057423\n",
      "Average loss at step 800 for last 100 steps: 0.465999576449\n",
      "Average loss at step 900 for last 100 steps: 0.4665778023\n",
      "\n",
      "EPOCH 5\n",
      "Average loss at step 100 for last 100 steps: 0.473842710853\n",
      "Average loss at step 200 for last 100 steps: 0.465623610318\n",
      "Average loss at step 300 for last 100 steps: 0.464327248037\n",
      "Average loss at step 400 for last 100 steps: 0.466024757326\n",
      "Average loss at step 500 for last 100 steps: 0.463789981306\n",
      "Average loss at step 600 for last 100 steps: 0.464974922836\n",
      "Average loss at step 700 for last 100 steps: 0.464186963439\n",
      "Average loss at step 800 for last 100 steps: 0.465779932141\n",
      "Average loss at step 900 for last 100 steps: 0.464908615053\n",
      "\n",
      "EPOCH 6\n",
      "Average loss at step 100 for last 100 steps: 0.471228752434\n",
      "Average loss at step 200 for last 100 steps: 0.463979724646\n",
      "Average loss at step 300 for last 100 steps: 0.463916065693\n",
      "Average loss at step 400 for last 100 steps: 0.463100931942\n",
      "Average loss at step 500 for last 100 steps: 0.463718650043\n",
      "Average loss at step 600 for last 100 steps: 0.46325374186\n",
      "Average loss at step 700 for last 100 steps: 0.462417589128\n",
      "Average loss at step 800 for last 100 steps: 0.462667659223\n",
      "Average loss at step 900 for last 100 steps: 0.462877951264\n",
      "\n",
      "EPOCH 7\n",
      "Average loss at step 100 for last 100 steps: 0.471185879111\n",
      "Average loss at step 200 for last 100 steps: 0.463871817291\n",
      "Average loss at step 300 for last 100 steps: 0.461288903356\n",
      "Average loss at step 400 for last 100 steps: 0.463138556778\n",
      "Average loss at step 500 for last 100 steps: 0.461805641353\n",
      "Average loss at step 600 for last 100 steps: 0.461411690414\n",
      "Average loss at step 700 for last 100 steps: 0.462100447714\n",
      "Average loss at step 800 for last 100 steps: 0.460773537159\n",
      "Average loss at step 900 for last 100 steps: 0.463051837087\n",
      "\n",
      "EPOCH 8\n",
      "Average loss at step 100 for last 100 steps: 0.469712789953\n",
      "Average loss at step 200 for last 100 steps: 0.460892218649\n",
      "Average loss at step 300 for last 100 steps: 0.460133855343\n",
      "Average loss at step 400 for last 100 steps: 0.45925313592\n",
      "Average loss at step 500 for last 100 steps: 0.459609957933\n",
      "Average loss at step 600 for last 100 steps: 0.459926410615\n",
      "Average loss at step 700 for last 100 steps: 0.459051845968\n",
      "Average loss at step 800 for last 100 steps: 0.45918114841\n",
      "Average loss at step 900 for last 100 steps: 0.461085427701\n",
      "\n",
      "EPOCH 9\n",
      "Average loss at step 100 for last 100 steps: 0.466978085041\n",
      "Average loss at step 200 for last 100 steps: 0.458726797104\n",
      "Average loss at step 300 for last 100 steps: 0.459468673468\n",
      "Average loss at step 400 for last 100 steps: 0.458695850968\n",
      "Average loss at step 500 for last 100 steps: 0.458720811307\n",
      "Average loss at step 600 for last 100 steps: 0.458215084076\n",
      "Average loss at step 700 for last 100 steps: 0.459488655329\n",
      "Average loss at step 800 for last 100 steps: 0.459476181567\n",
      "Average loss at step 900 for last 100 steps: 0.457595551908\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7ff00028da58>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xl8VPW9//HXZyYLS4BASICwBWQX2UQEVwRR1Apu16LWSltre1uqrdVWu/+0y22919vNLl7F6vVatbiAgiJurRtIkICERRaBJBASCDtk//z+mCFmmSQDJAQm7+fjkUfnnDlnznem43u+fL7fc465OyIi0joEWroBIiJy4ij0RURaEYW+iEgrotAXEWlFFPoiIq2IQl9EpBVR6IuItCIKfRGRVkShLyLSisS1dANq69q1q2dkZLR0M0RETinLli3b6e6pjW130oV+RkYGmZmZLd0MEZFTipltiWY7lXdERFoRhb6ISCui0BcRaUUU+iIirYhCX0SkFVHoi4i0Igp9EZFWJGZC/2BJOQ8u+oTlW3e3dFNERE5aMRP6JeWV/P6N9azI2dPSTREROWnFTOjHBw2A8krd6F1EpD4xFPqht1JaUdnCLREROXnFXOiXlaunLyJSn6hC38ymmtk6M9tgZvfUs831ZrbazLLN7Klq628xs/Xhv1uaquG1BQNGMGCUqacvIlKvRq+yaWZB4CFgCpALLDWzee6+uto2A4F7gXPdfbeZpYXXdwF+CowFHFgW3rdZptjEBxX6IiINiaanPw7Y4O6b3L0UeBqYXmubrwIPHQlzdy8Ir78UWOTuReHnFgFTm6bpdcUHA6rpi4g0IJrQ7wnkVFvODa+rbhAwyMzeM7PFZjb1KPZtMvHBgHr6IiINaKqbqMQBA4GJQC/gX2Z2RrQ7m9ltwG0Affr0OeZGxAdNA7kiIg2IpqefB/SuttwrvK66XGCeu5e5+6fAJ4R+BKLZF3d/2N3HuvvY1NRG7/ZVr/hggLJK9fRFROoTTegvBQaaWT8zSwBmAPNqbfMioV4+ZtaVULlnE7AQuMTMOptZZ+CS8LpmkRAMUFahnr6ISH0aLe+4e7mZzSIU1kFgtrtnm9l9QKa7z+OzcF8NVAB3u/suADO7n9APB8B97l7UHG8Ewj39cvX0RUTqE1VN390XAAtqrftJtccO3Bn+q73vbGD28TUzOvFxmrIpItKQmDkjFzRlU0SkMTEX+urpi4jUL8ZC3zSQKyLSgBgLffX0RUQaEoOhr56+iEh9Yir0E9TTFxFpUEyFvq6yKSLSsBgLfZ2cJSLSkNgK/bgAparpi4jUK6ZCXzV9EZGGxVTox+l2iSIiDYqp0I+PU09fRKQhsRX64Xn6oeu/iYhIbTEV+glBA6C8UqEvIhJJTIV+fDD0dlTiERGJLDZDX/fJFRGJKLZCPy70dnRNfRGRyGIq9I/U9FXeERGJLKrQN7OpZrbOzDaY2T0Rnp9pZoVmlhX+u7Xac78xs2wzW2Nmvzcza8o3UF1cQDV9EZGGNHqPXDMLAg8BU4BcYKmZzXP31bU2fcbdZ9Xa9xzgXGBEeNW7wIXA28fZ7oiOlHcU+iIikUXT0x8HbHD3Te5eCjwNTI/y9R1oAyQAiUA8sONYGhqNz8o7GsgVEYkkmtDvCeRUW84Nr6vtWjNbaWZzzKw3gLt/ALwFbA//LXT3NcfZ5nppyqaISMOaaiD3JSDD3UcAi4DHAcxsADAU6EXoh2KSmZ1fe2czu83MMs0ss7Cw8JgbodAXEWlYNKGfB/SuttwrvK6Ku+9y95Lw4iPAmeHHVwOL3f2Aux8AXgEm1D6Auz/s7mPdfWxqaurRvocqR0K/VPP0RUQiiib0lwIDzayfmSUAM4B51Tcwsx7VFqcBR0o4W4ELzSzOzOIJDeI2W3knIU5TNkVEGtLo7B13LzezWcBCIAjMdvdsM7sPyHT3ecDtZjYNKAeKgJnh3ecAk4CPCQ3qvuruLzX92whReUdEpGGNhj6Auy8AFtRa95Nqj+8F7o2wXwXwteNsY9Q0T19EpGGxdUZuuLyjWyaKiEQWU6F/pLxTrp6+iEhEMRn6Ku+IiEQWk6Gv8o6ISGQxFfoJVdfTV09fRCSSmAr9eM3TFxFpUGyFvmr6IiINiqnQjwtoyqaISENiKvTNjPigqacvIlKPmAp9CJV4NE9fRCSymAx93URFRCSymAz9UvX0RUQiirnQTwia5umLiNQj5kI/Pi6ggVwRkXrEXuirpi8iUq+YC/24gKmmLyJSj5gL/QSVd0RE6hVzoR+ap6/yjohIJFGFvplNNbN1ZrbBzO6J8PxMMys0s6zw363VnutjZq+Z2RozW21mGU3X/LrigyrviIjUp9F75JpZEHgImALkAkvNbJ67r6616TPuPivCSzwB/MLdF5lZEtCsiRwfDHCgpLw5DyEicsqKpqc/Dtjg7pvcvRR4GpgezYub2TAgzt0XAbj7AXc/dMytjUJCUDV9EZH6RBP6PYGcasu54XW1XWtmK81sjpn1Dq8bBOwxs+fNbLmZPRD+l0MNZnabmWWaWWZhYeFRv4nq4oMByspV0xcRiaSpBnJfAjLcfQSwCHg8vD4OOB+4CzgL6A/MrL2zuz/s7mPdfWxqaupxNUQnZ4mI1C+a0M8Deldb7hVeV8Xdd7l7SXjxEeDM8ONcICtcGioHXgTGHF+TGxavefoiIvWKJvSXAgPNrJ+ZJQAzgHnVNzCzHtUWpwFrqu2bbGZHuu+TgNoDwE0qXjV9EZF6NTp7x93LzWwWsBAIArPdPdvM7gMy3X0ecLuZTQPKgSLCJRx3rzCzu4A3zMyAZcD/NM9bCYmPM83TFxGpR6OhD+DuC4AFtdb9pNrje4F769l3ETDiONp4VHRpZRGR+sXcGbmasikiUr+YC31dZVNEpH4xGfoVlU5FpYJfRKS22Av9OANQiUdEJILYC/1A6C0p9EVE6oq90A8e6emrvCMiUlvshX5c6C2Vq6cvIlJH7IV+MPSWNFdfRKSumAv9hOCRmr7KOyIitcVc6McHNZArIlKfGAz90EBuablCX0SkttgL/Tj19EVE6hN7oR9QTV9EpD6xF/pBnZErIlKf2At9lXdEROoVc6GvKZsiIvWLudDXlE0RkfpFFfpmNtXM1pnZBjO7J8LzM82s0Myywn+31nq+o5nlmtkfm6rh9VFNX0Skfo3eLtHMgsBDwBQgF1hqZvPcvfYNzp9x91n1vMz9wL+Oq6VRqroMg+bpi4jUEU1Pfxywwd03uXsp8DQwPdoDmNmZQDfgtWNr4tFJiFNNX0SkPtGEfk8gp9pybnhdbdea2Uozm2NmvQHMLAD8F3DXcbc0SnEBlXdEROrTVAO5LwEZ7j4CWAQ8Hl7/DWCBu+c2tLOZ3WZmmWaWWVhYeFwN0ZRNEZH6NVrTB/KA3tWWe4XXVXH3XdUWHwF+E348ATjfzL4BJAEJZnbA3e+ptf/DwMMAY8eOPa66jKZsiojUL5rQXwoMNLN+hMJ+BnBj9Q3MrIe7bw8vTgPWALj7TdW2mQmMrR34TU1TNkVE6tdo6Lt7uZnNAhYCQWC2u2eb2X1AprvPA243s2lAOVAEzGzGNjcoGDACptAXEYkkmp4+7r4AWFBr3U+qPb4XuLeR1/gb8LejbuExiA8GdOcsEZEIYu6MXAjV9cvKVdMXEaktJkM/Pi6g8o6ISASxGfpBU+iLiEQQk6EfF1BNX0QkkpgM/YS4AOWapy8iUkdMhr7KOyIikcVo6GsgV0QkkpgN/VKVd0RE6ojJ0A/N01dPX0SktpgM/fg41fRFRCKJzdBXTV9EJKKYDP3QPH3V9EVEaovJ0E+IM8rV0xcRqSMmQ1/lHRGRyGI49FXeERGpLWZDX9feERGpKyZDP0GXYRARiSgmQz9eJ2eJiEQUVeib2VQzW2dmG8yszo3NzWymmRWaWVb479bw+lFm9oGZZZvZSjP7fFO/gUhCN1FRTV9EpLZG75FrZkHgIWAKkAssNbN57r661qbPuPusWusOAV909/Vmlg4sM7OF7r6nKRpfn/iAUVpRibtjZs15KBGRU0o0Pf1xwAZ33+TupcDTwPRoXtzdP3H39eHH24ACIPVYGxut+GDobVVUqrcvIlJdNKHfE8iptpwbXlfbteESzhwz6137STMbByQAG4+ppUchPi70tlTiERGpqakGcl8CMtx9BLAIeLz6k2bWA/hf4EvuXmeE1cxuM7NMM8ssLCw87sYc6elr2qaISE3RhH4eUL3n3iu8roq773L3kvDiI8CZR54zs47AfOCH7r440gHc/WF3H+vuY1NTj7/6kxAM1fE1bVNEpKZoQn8pMNDM+plZAjADmFd9g3BP/ohpwJrw+gTgBeAJd5/TNE1u3JGevkJfRKSmRmfvuHu5mc0CFgJBYLa7Z5vZfUCmu88DbjezaUA5UATMDO9+PXABkGJmR9bNdPespn0bNVWFfrlq+iIi1TUa+gDuvgBYUGvdT6o9vhe4N8J+TwJPHmcbj9qRgVzV9EVEaorNM3IDqumLiEQSm6EfLu+Ua8qmiEgNsRn6Ku+IiEQUm6GvKZsiIhHFZOgnaMqmiEhEMRn6mqcvIhJZTId+qebpi4jUEJOhnxCnmr6ISCQxGfpxAZV3REQiicnQPzJlU/P0RURqis3QD0/Z1Dx9EZGaYjL0NWVTRCSymAx9TdkUEYksxkNfNX0RkepiNPTDNf1y9fRFRKqLydA3M+KDpvKOiEgtMRn6EJqrr9AXEakpZkM/1NNXTV9EpLqoQt/MpprZOjPbYGb3RHh+ppkVmllW+O/Was/dYmbrw3+3NGXjG5IQp56+iEhtjd4j18yCwEPAFCAXWGpm89x9da1Nn3H3WbX27QL8FBgLOLAsvO/uJml9A+KDCn0Rkdqi6emPAza4+yZ3LwWeBqZH+fqXAovcvSgc9IuAqcfW1KMTCn2Vd0REqosm9HsCOdWWc8PrarvWzFaa2Rwz6300+5rZbWaWaWaZhYWFUTa9YfFB02UYRERqaaqB3JeADHcfQag3//jR7OzuD7v7WHcfm5qa2iQNig8GKNM8fRGRGqIJ/Tygd7XlXuF1Vdx9l7uXhBcfAc6Mdt/mooFcEZG6ogn9pcBAM+tnZgnADGBe9Q3MrEe1xWnAmvDjhcAlZtbZzDoDl4TXNbu4gKZsiojU1ujsHXcvN7NZhMI6CMx292wzuw/IdPd5wO1mNg0oB4qAmeF9i8zsfkI/HAD3uXtRM7yPOjR7R0SkrkZDH8DdFwALaq37SbXH9wL31rPvbGD2cbTxmCTEBThYUn6iDysiclKL4TNyNWVTRKS2GA59XXBNRKS2mA39rkmJ5BQd4nBpRUs3RUTkpBGzoX/lyHQOllawMDu/pZsiInLSiNnQH5fRhV6d2zJnWW5LN0VE5KQRs6EfCBjXjunFext3sm3P4ZZujojISSFmQx/g2jG9cIcXljfvScA5RYe49fGl7DpQ0vjGIiItKKZDv09KO8b168Jzy3Jxb77pm89m5vD6mgKeXLy12Y4hItIUYjr0Aa4b04tNOw+yPGdPsx1j0eodADz14RZNExWRk1rMh/7lI3rQNj5YNaC7+2Ap/8jMYUUT/QjkFB1ibf5+zh2Qwo59JbyWvaNJXldEpDlEdRmGU1lSYhxTh3fnpaxt5O0+zHsbdlJe6SS3i2fB7eeTntz2uF7/9TWhkL9v+nBmPvYhj3+wmStG9Gh4JxGRFhLzPX2AGWf1Zn9JORsLD3Dr+f155ItjKSuv5NtPZ1F+nOWYRat3MCAtidNSk7h5fF8+/LSItfn7mqjlIiJNq1WE/tn9U1j6w4t553sXcc9lQ7h4WDd+fvVwPtxcxO/f3HDMr7v3UBlLPi1iyrBuAFw/tjeJcQGe+GBLUzVdRKRJxXx554jUDok1lq8e3Yt31+/iD2+uZ3z/LnTr2IYPNu5i2ZbddGwTx+k9O3F6ekcGpnUgIS7yb+PbnxRQUelVoZ/cLoHpo9J54aM8vj91CJ3axjf7+yqrqGR/cTld2ic0+7FE5NTXakI/kvumn87ynN3c9MgSjszoTO2QyMGSch4P99bNIKV9Ij06taFnclu+NXkAp6d3AkKlna5JiYzqlVz1ml+ckMGzmbk8sHAt91w2lKTE5v2I/3vRJzz14VYW3zuZNvHBZj2WiJz6WnXot0+M469fOJPZ733KyF7JjO+fQt+UdrjD5l0HWbVtHxsLDrBjXzH5+4pZurmI6//yAX/6wplM6J/CP9cVcsWIHgQCVvWaw3t24poxPXly8VbmZW3jlnMymHlOBilJiQ205NhUVjovLs9jz6EyFm/axcTBaU1+DBGJLa069AEGduvAr64ZUWOdGfRPTaJ/alKN9Tv2FTPzsaV8+W9LuW5ML/aXlFeVdqp78PpR3DIhgz+/vZE/vrWB/128hVfuOJ8enY5vplBtWbl72La3GIC31hYo9EWkUVEN5JrZVDNbZ2YbzOyeBra71szczMaGl+PN7HEz+9jM1phZxLtrnSq6dWzDs18bzzmnpfBMZg5t44OcO6BrxG1H9k7mLzefycvfOo/isgp+Oje7ydszf+V2EoIBxvXrwhtrC5r1rGMRiQ2Nhr6ZBYGHgMuAYcANZjYswnYdgDuAJdVW/xuQ6O5nAGcCXzOzjONvdsvp0Cae2TPP4ivn9eObF53WaB399PRO3DF5EK+t3tGkl3murHQWfLydCwZ1ZdrIdHJ3H2ZDwYEme/1oHC6t0P0KRE4x0fT0xwEb3H2Tu5cCTwPTI2x3P/BroLjaOgfam1kc0BYoBU75SezxwQA//twwZk0aGNX2t57fjyHdO/DTudnsLy47qmMV7i/hkXc28d1nV7Cv2r7Lc/awfW8xV4zowUVDQmWdN9cWHNVrH6+vPpHJ159cdkKPKSLHJ5rQ7wnkVFvODa+rYmZjgN7uPr/WvnOAg8B2YCvwn+5edOzNPTXFBwP86poz2LG/mP967RMOlJTz1JKtTPvju1z5h3fZfbC0zj7vb9zJl/+2lPG/eoOfz1/Dcx/l8qMXVlWVcOav3E5CXICLh3ajZ3JbhnTvUCf0NxYe4KOtu5vlPRXsK+a9jTv5YOMuisvU2xc5VRz3yVlmFgAeBL4b4elxQAWQDvQDvmtm/SO8xm1mlmlmmYWFhcfbpJPS6D6duXl8Xx7/YDNn/+J1fvDCx5SUVbJux36+9LelHCotr9r2H5k5fOGRJazeto/bLujP63dewHenDGLeim08/1HeZ6Wdgal0aBM6F+CiIWlkbtnN3sOhfw3sPljKDQ8v5po/vc8DC9ce95nHtS1cvQN3KK2oZPnW5ruYnYg0rWhCPw/oXW25V3jdER2A4cDbZrYZGA/MCw/m3gi86u5l7l4AvAeMrX0Ad3/Y3ce6+9jU1NRjeyengLsvHcy4jC5cfkYPnv/GObz67fP5/YzRrMzdw9ef/IjS8kpmv/spd89ZyTmndeWN717I96cOYUBaB75x0QDG9evCT+au4oXleeTvK+Zz1a7xM2lIGhWVzjvrC3F3fvDCx+w+VMrlZ3Tnobc28oVHl1Cwv7iB1h2dV1dtp2dyWwIGizftarLXFZHmFU3oLwUGmlk/M0sAZgDzjjzp7nvdvau7Z7h7BrAYmObumYRKOpMAzKw9oR+EtU38Hk4ZHdrE88zXJvDAv41kTJ/OmBlTh3fnl1efwb8+KeTKP7zLfS+vZurp3Xl05ljaVzuxKxgwfvv5UQQDxl1zVpAQF2Dy0M+maI7unUyntvG8ubaA5z/K45VV+dw5ZTB/uulMHrhuBFk5e7j8d+/y6LufHvW4Qm27D5ayeFMRV41OZ1h6R4W+yCmk0dB393JgFrAQWAM86+7ZZnafmU1rZPeHgCQzyyb04/GYu6883kbHmhnj+vC9qYNZt2M/153Ziz/eOJrEuLqzgtKT2/Lra0fgDhcO+qy0AxAXDDBxcCpvrCngp/OyGZfRhdsuCFXS/m1sb1785rlkpLTj/pdXM/6Xb/Czedms2b7vmKZ5Llq9g4pK57LhPRjfL4XlOXtU1xc5RdjJNrd77NixnpmZ2dLNaBFbdh2kT5d2mFmD283NymN4z06cVuvksblZedzxdBZJiXG8csf59O7Srs6+K3P38Nh7m3l55TbKKpz0Tm2YOCSNy4f34LyBkc85qO1Lj33I+oIDvPO9i3hjTQG3PpHJ07eNZ3z/lOjfrIg0KTNb5u51yue1tYqrbJ4q+qa0bzTwAaaP6lkn8AEmDkqjb0o7fnnNGREDH2BEr2T++/OjeP+eyfz62jM4o1cnXlyexxceXcKPX1xFaXnDA777ist4d8NOpp7eHTPjrH5dMIMlm1rdpCyRU1KrvwxDLOnULp5/3n1RVNumdkjk82f14fNn9aGkvIIHX/uEv/5rE+vy9/PQTWOqrkq651AplU7VVTzfWltAWYVz2RndQ8dsG8+wHqG6/h1Ed96CiLQchb6QGBfk3suHMiy9I99/biVX/uFdBnfvwLr8/eTvKyYuYFw5Mp3bLujPKx/n061jIqN7d67a/+x+Kfzfki2UlFdEHItoDlk5e+jbpR2dT/AlpSsqnec+ymXayHRd1VROSSrvSJXpo3oy5+vnkJKUQMH+EiaclsK9lw3h5gl9WZidz2W/e4fXVudz6enda1xZdHz/LpSUV7IiZ+8JaeeOfcVc9+f3ue/l1SfkeNW9lp3P9+as5JmlOY1vLHISUk9fahjesxPzbz+/zvpvTx7Ek0u2sODj7cw4q0+N58ZV1fV3Ma5fl2Zv49Mf5lAePkHtZ1eeTqd2zX+zmiPmZm0DQjOYbjkn44QdV6SpqKcvUenULp5vXjSA+befz7D0jjWeS26XwJDuHVn8aeT5+hWVzt5Dx3duwBHlFZX8/cOt9OvanpLySl7Mymt8pyay93AZb64roE18gMWbdjXZexI5kRT60iTG9+/Csi27ydxcVHVhuJyiQ/zXa+s49z/eZNT9r3Hj/yxmzrJcDpSUN/Jq9XtjbQH5+4q557IhDO/Zkb9/uPWEXVJ64ap8Sssr+f7UIZRXOm+tO7EXuFuYnc+5//Emew7VvVZTc/vz2xv5d11cLyaovCNN4qLBaTz23mau+8sHAKR1SKRgfwkBC51Ids2Ynsz/eDt3/WMFP3rxY3p1bkdy23iS28Vz/sDUqEslTy7eQo9ObZg8JI3C/SX86MVVrMjdy6jeyY3vfJzmrsijb0o7vjghgz+9vZHXVudz1eieje/YRP7yz43k7TnM/I+3c9PZfU/Ycd2dJz7YzPa9xWwoOMCAtLrTheXUodCXJnHBoFTev2cSa/P3sS7/AOsL9tMvpT3Xje1Vdcewuy8dzEdbd/Pyyu3k7y1mz6EyNhUe5PU1BXRun8C0kekNHmPzzoO8s34n37l4EHHBANNHpfOL+Wt4ZunWZg/9HfuKeX/jLr41aSDBgDFlWDdeXJ5HcVnFCZnFk71tb9WF7eZmbTuhob8qbx/bw3dom7diG3dOGXTCjg2hkwFH9k7m2xef2OPGKoW+NJn05LakJ7dl0pC6t5AEMDPO7NuFM/t+NthbVlHJjIcXc+9zKxme3rHGLSrfWV9IaXklEwenEQwYT324lWDAmDEudP2/Dm3iuWJED+ZlbeNHVwyrca2ixhwureCn81bRvWMbvjV5IPHBhiudL63YhjtMHxX6YbpkWDeeWrKV9zfurPf9NqWnlmwlMS7AF8b35dF3PyVvz2F6Jjft7Tfrs2h1PgGDoT06Mi8rj+9cPDCqkwibwqq8vby1rpC1+fu5Y/KJO24sU01fWlR8MMAfbhhNQlyAbz61nOKyCvYeKuOOp5dz86Mf8pXHM7ngN2/xp7c38I/MHC4Z1o1uHdtU7X/DuN4cLK3gpRXb6rz2gZJylm0pYkPB/hrrdx8s5cZHFvOPZbn8/s0NXPfn99m882CD7ZybtY0zql36YsJpKSQlxvFa9o4m+BQadqCknBeX53HlyHS+OCHUw4/0fpvLa6t3MLZvF744oS+bdx1iZe6JmZoL8GxmaGrs9r3FrM3f38jWEg319KXFpSe35cHrR/Glvy1l1lMfkb1tHwX7S7hzyiAGdUvi8fe38JtX1wHwhfE1yxpj+nRmYFoSv319fdVNZMornY2FB9iy61DVdhcNTuXfJw6gZ+e2fPHRJeTsPsyfbzoTd+ee5z/mit+/w0+uHMY1Y3rV6fVvLDzAx3l7+dEVQ6vWJcYFmTg4ldfXhC4+Fww0Xw90blYeB0sruOnsPvRNac/oPsm8uDyPr194WrMd84icokOszd/PDy8fytTTe/DjF7OZm7WNkSdgDKW4rIIXl+cxoX8KH2zaxVvrChjao2PjOzahrJw9DOneIaZOxFPoy0nhoiFpfP3C0/jLPzfSP7U9L3zjHEb0CgXL1OE9+GTHftbm7+ec02pe1M3M+M6UQfzhzQ1sLTpUtW5Yj45cN6YXQ3t0ZG3+Ph57bzPX//UDEuMCJMQF+N8vj+Ps8AXiRvZO5tvPZPH95z7mN6+u46rRPblmTE8Ol1awbMtuXs3Oxww+N6LmmMMlp3fn5ZXbycrZXaNkdUR5RSXBgB1XScLdeXLxVob16Fg1bjF9ZDo/e2k16/L3M7h7h2N+7WgsWh36l8yUYd3o1C6eiYNTeXnlNn54xdBm/aGD0GylfcXlzJo0gH3FZby1toBvTBzQrMesLnvbXq566D3uvnQw37zoxB23uSn05aRx1yWDOCujM+ec1pW2CTV7VoO6dWBQt8gBd/kZPbj8jB4RnwO4eFg3bj2/P//IzOGNtQXcc9kQhnT/rMeYntyWv391PG+vK+Afmbk88cFmHn3306rnM1La8f2pQ+jeqU2N1504OJX4oPHs0lxG9e5cFYLuzrwV2/h/L61mdO9kfn/D6KjHG97fsJOnPgwNTF8xogf5e4tZs30fv7h6eNWPxxUj0rl//hrmZuXxvalDonrdY7Vo9Q4GpiWR0bU9EDpr+7XVO1iyaRfnDIjuqqzH6tnMHHp1bsuE/ilMGpLGQ29tYO+hshN2Mt5j720G4NVV+Qp9keYQFwwweWjzDIq2iQ9y84QMbp6QEfH5YMCYPLQbk4d2Y9eBEhat3kGX9gmM6duZrkmJEfc/9TTDAAAM6UlEQVTp2CaeK0em80xmDks3F3HHxQM5u18KP567ikWrdzCoWxJvrSvg+r9+wOyZZ1WNRWTl7GHOshz6dmnPlGHdyOjanp0HSvjF/DW8sDyPDm3ieHnldn4+fw1d2ifQPiHI9FGfTQ1N7ZDIuQO6MjdrG3ddMrjGJTGa0p5DpXy4uYivXfDZHU4nD02jfUKQuVnbmjX0c4oO8d6GXXzn4kEEAsbEwWn84c0N/HN9YaOzvJpC4f4S5mVto1PbeD7O28u2PYdJP0ED581NoS9SS0pSIjPG9Wl8Q+A/rxvJJcO68dvX13PH01mYhQanf3D5EL5yXn/+9Ukh33zqI65+6D2+M2UQc5blsuTTIhLjApSUV/KLBWsYmJZEwf4SDpWWc/ukAXzjogFs31vM/JXbeDU7nylDu5NU618KV41K585nVzD7vU/JSGlPfFyAtA6JDOneoclmuLy1roCKSmfKsM9+iNvEB7l0eHcWrNrOfVed3mwX2PtHZg5mcN3YXgCM6p1M53bxvL224ISE/lNLtlJaUclD/zaGrz6RyetrdvDFejoMpxqFvshxCASMqcN7cMmw7ryanc/iTbu45ZyMqlk+Fw1J49mvTeDLf1vK3XNW0qNTG350xVBmjOvD7oOlvL5mB6+v2UGfLu249/LQ/ZAB+nVtz6xJA5k1KfLlqi85vTsd2mTz8/lraqzvmdyWS0/vzgWDupJTdIgPN+9m6adFlFZUMqR7B4Z078jp6R2ZODiVlHr+BXPEotU7SOuQyMheNQdtrxrVk+c/yuOBV9fxwyuG1viRqah0thYdol+4HHQsKiqdOctyOX9gatW01GDAuHBQKm9/UtjsA+cl5RX87+ItTBycypRh3eif2p7XshX6IlJNIGD1ji0M79mJl751Hsu37mHSkDQS4kKzg5IS4/jSuf340rn9jvp4SYlx/PPuiyjcX0JpeSWlFZVsLDjAwux8nlyyhdnvhcYkunVMZFy/FNrFB1mbv4+nPtxCcVklAQtdKO+y4T3ompRI4f5iCvaXULi/hKKDpew6WEr2tr1cd2bvOuWj8wd25ebxfXnk3U8JBo17pg7BzMjbc5jvPJ3Fh5uLOHdACt+9ZDBj+nSO1Px6uTu/ff0Ttu0t5gfVZktB6Af0xaxtrMzdw+ijfN2jMX/ldnYeKOHL4f9fpgzrxqPvfMrew2V0anviLu7XXKIKfTObCvwOCAKPuPt/1LPdtcAc4KzwjdExsxHAX4GOQGX4ueImaLvIKaNbxzZMHd69SV+zS/uEqpvbAJzZtzPXn9WbAyXlfLRlNxkp7endpW2dnvia7ftYmJ3PK6vy+em87Krn4gJG16REurRPICUpgc+NSOcr59X9QTIz7pt+Oo7z139uImDGGT07cc9zK6modL58bj/mZuVxzZ/eZ/KQNKaNSue01KSq3v+ST3fxzvqdrMjZw8TBaXz5vH4kJcZRXlHJD19YxTOZOVw7pheXDa/5A3rBwFQCFrqRT3OFvrsz+71PGZCWxPnh24deMqw7f/3nJt5eV1BjbOVU1eg9cs0sCHwCTAFyCd3g/AZ3X11ruw7AfCABmOXumWYWB3wE3OzuK8wsBdjj7vXeRbs13yNX5ETbVHiAkvJK0jok0rldwlENCldWOj+eu4r/W7IVCE19/f2MUfRNac/BknIee+9T/vqvTewv/uwCe8GAUVHpJMYFGJCWRPa2fXRuF8/XLzyNxZt28da6Qr41aQB3ThkUcWzi2j+/T3FZBS/NOq9ZBrDf37CTGx9Zws+vGl51TkhlpTPul29wdv8uPHTjmCY/ZlOJ9h650fT0xwEb3H1T+IWfBqYDte9gcT/wa+DuausuAVa6+woAd4987V0RaRH9I9xrOVqBgHH/9OGktE8gGAjwjYtOqzqxrX1iHLMmDeSrF/Rn885DbCo8wKadBykuq2B8/xTO7NuZNvFBVuTs4b8WfcKvXllLwOAXVw9v8LpCl57ejV8uWMu4X77BpCGpTBqSxgWDUmmXcPyV6jnLcvnxi6FLc1wz5rMefSBgTBmWxrysbSf07nDNJZpPqidQ/TZBucDZ1TcwszFAb3efb2bVQ38Q4Ga2EEgFnnb33xxnm0XkJBEIGHdeMrje5xPjggzu3qHek8hG9k7miS+PY9mWItxhbEbDN+H5ynn9Se2QyBtrCnhlVT7PZubSNj7IxcO6ceWIHgzv2Ync3YfZvOsgW3cdInf3IXJ3HyZvz2G6tE/g8jN6cOWIdPqktKt6zYMl5fx47iqe/yiP8f278LsZo+v8iFwyrDt//zCHDzbuYuLgNHYdKGHVtn0MT+/Y6IB4dbm7D7Fsy+5qn0+ACwel1TkvpTkd98+jmQWAB4GZ9bz+ecBZwCHgjfA/Qd6o9Rq3AbcB9OkT3VQ5EYkdkc5ojiQYMK4e3YurR/eirKKSpZuLmL9yO6+syq9zPaJgwOjRqQ09k9sy4bQUNu88yAML1/HAwnUMSEvCCAX+nsNlHC6r4I7JA7l98sCIM4MmnJZCu4QgDy76hN++vp4VuXtwB7PQpUAmD01jQGoSlQ6V7iQEAwzu3oFenUNjKlt3HeKhtzbw3Ee5lFfWLKkPTEviDzeOrnHCYHOKpqY/AfiZu18aXr4XwN1/FV7uBGwEDoR36Q4UAdOAAcBl7n5LeNsfA8Xu/kB9x1NNX0SOVllFJe9v3MXWokP06dKOjJR2pCe3rXMdpdzdh1jw8XYWbwqdK9E+MY6kxDguG9696rIc9bnzmSxeyMpjZK9kLhqcxqg+yXy0ZTdvrN3Bqrx9EfdJbhfPaalJZOXsIRgwbjirNzPG9SExLkClh8ZUfvDCKvYXl/Hjzw3jprP7HPN5FtHW9KMJ/ThCA7mTgTxCA7k3unt2Pdu/DdwVHsjtDLxBqLdfCrwK/Le7z6/veAp9ETkZlZRXUFxWGXHa5o59xRTuLyFgRiAAh0orWL1tH6vy9rI2fz+jeifz7xNPq3GF2CMK95dw57NZvLN+J5ef0Z0/3jDmmAapm2wg193LzWwWsJDQlM3Z7p5tZvcBme4+r4F9d5vZg4R+KBxY0FDgi4icrBLjgvUO4nbr2KZOoEd7jkJqh0Qe/9I4/ued0Eyn5rqsxhGN9vRPNPX0RUSOXrQ9fd1ERUSkFVHoi4i0Igp9EZFWRKEvItKKKPRFRFoRhb6ISCui0BcRaUUU+iIirchJd3KWmRUCW47jJboCO5uoObFAn0dN+jzq0mdS06n6efR199TGNjrpQv94mVlmNGeltRb6PGrS51GXPpOaYv3zUHlHRKQVUeiLiLQisRj6D7d0A04y+jxq0udRlz6TmmL684i5mr6IiNQvFnv6IiJSj5gJfTObambrzGyDmd3T0u1pCWbW28zeMrPVZpZtZneE13cxs0Vmtj78v9Hd3SFGmFnQzJab2cvh5X5mtiT8XXnGzBJauo0nipklm9kcM1trZmvMbIK+H/ad8H8vq8zs72bWJpa/IzER+mYWBB4CLgOGATeY2bCWbVWLKAe+6+7DgPHAN8Ofwz3AG+4+kNDtK1vbj+IdwJpqy78mdNvOAcBu4Cst0qqW8TvgVXcfAowk9Lm02u+HmfUEbgfGuvtwQncHnEEMf0diIvSBccAGd9/k7qXA08D0Fm7TCefu2939o/Dj/YT+g+5J6LN4PLzZ48BVLdPCE8/MegFXAI+Elw2YBMwJb9JqPg8z6wRcADwK4O6l7r6HVvz9CIsD2obvB94O2E4Mf0diJfR7AjnVlnPD61otM8sARgNLgG7uvj38VD7QrYWa1RJ+C3wPqAwvpwB73L08vNyaviv9gELgsXC56xEza08r/n64ex7wn8BWQmG/F1hGDH9HYiX0pRozSwKeA77t7vuqP+eh6VqtYsqWmX0OKHD3ZS3dlpNEHDAG+LO7jwYOUquU05q+HwDh8YvphH4Q04H2wNQWbVQzi5XQzwN6V1vuFV7X6phZPKHA/z93fz68eoeZ9Qg/3wMoaKn2nWDnAtPMbDOhkt8kQjXt5PA/5aF1fVdygVx3XxJenkPoR6C1fj8ALgY+dfdCdy8Dnif0vYnZ70ishP5SYGB4xD2B0EDMvBZu0wkXrlc/Cqxx9werPTUPuCX8+BZg7oluW0tw93vdvZe7ZxD6Trzp7jcBbwHXhTdrTZ9HPpBjZoPDqyYDq2ml34+wrcB4M2sX/u/nyGcSs9+RmDk5y8wuJ1S/DQKz3f0XLdykE87MzgPeAT7msxr2DwjV9Z8F+hC6gun17l7UIo1sIWY2EbjL3T9nZv0J9fy7AMuBL7h7SUu270Qxs1GEBrUTgE3Alwh1/lrt98PM/h/weUKz35YDtxKq4cfkdyRmQl9ERBoXK+UdERGJgkJfRKQVUeiLiLQiCn0RkVZEoS8i0ooo9EVEWhGFvohIK6LQFxFpRf4/sJn2X/f3WQwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff0042c76d8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def train_network(num_epochs, num_steps, state_size=4, verbose=True):\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        training_losses = []\n",
    "        for idx, epoch in enumerate(gen_epochs(num_epochs, num_steps)):\n",
    "            training_loss = 0\n",
    "            training_state = np.zeros((batch_size, state_size))\n",
    "            if verbose:\n",
    "                print(\"\\nEPOCH\", idx)\n",
    "            for step, (X, Y) in enumerate(epoch):\n",
    "                tr_losses, training_loss_, training_state, _= \\\n",
    "                    sess.run([losses, \n",
    "                              total_loss,\n",
    "                              final_state,\n",
    "                              train_step],\n",
    "                            feed_dict={x:X, y:Y, init_state:training_state})\n",
    "                training_loss += training_loss_\n",
    "                if step % 100 == 0 and step > 0:\n",
    "                    if verbose:\n",
    "                        print(\"Average loss at step\", step,\n",
    "                              \"for last 100 steps:\", training_loss/100)\n",
    "                    training_losses.append(training_loss/100)\n",
    "                    training_loss = 0\n",
    "    return training_losses\n",
    "training_losses = train_network(10, num_steps, state_size=state_size)\n",
    "plt.plot(training_losses)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
